# coding: utf-8

__author__ = 'wangtao'


from urllib import urlencode
from urllib2 import urlopen, Request
from HTMLParser import HTMLParser
from BeautifulSoup import BeautifulSoup
from xml.dom.minidom import parse, parseString

from htmlentitydefs import name2codepoint


f = urlopen('http://xnny.net/')
inputStrWin = f.read()      # 整个页面的信息
soup = BeautifulSoup(inputStrWin)

try :
#    string = str(soup.html.body.contents[1])
    string = str(soup.html)

except :
    string = ''
string = '''<html lang="zh-CN">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link href="http://localhost/static/favicon.ico" rel="shortcut icon" type="image/x-icon" />
<link href="http://localhost/static/style/style.css?v=20101102" rel="stylesheet" type="text/css" media="screen" />
<title>我的订阅</title>
<style>
#news-bottom-list { text-align:center; }
</style>
</head>
<body>
<div></div>
ssfssdsfsfs
<div></div>
</body>
</html>'''



#if string != '':
#    dom = parseString(string)
#    element = dom.getElementsByTagName('body')
#    print(dom.getElementsByTagName('body')[0].nodeType)
##    print(dom.getElementsByTagName('body')[0].firstChild.data)
#    print(dom.getElementsByTagName('body')[0].firstChild.nodeType)
#    print(dom.getElementsByTagName('body')[0].firstChild.nodeValue)

#    if dom.getElementsByTagName('body')[0].nodeName and dom.getElementsByTagName('body')[0].nodeName > '' :
#        print('xx')



def getContent__manualSelection():
    _selection = selGetSelection(inputStrWin)
#    _range = $R.sel.getRange(_selection)
#    _html = $R.sel.getRangeHTML(_range)
#    _text = $R.sel.getRangeText(_range)

parsingOptions = {
    '_elements_ignore': 			'|button|input|select|textarea|optgroup|command|datalist|--|frame|frameset|noframes|--|style|link|script|noscript|--|canvas|applet|map|--|marquee|area|base|',      # 这些元素被忽略
    '_elements_ignore_tag': 		'|form|fieldset|details|dir|--|center|font|span|',  # 这些标签忽略
    '_elements_self_closing': 		'|br|hr|--|img|--|col|--|source|--|embed|param|--|iframe|',
    '_elements_visible': 			'|article|section|--|ul|ol|li|dd|--|table|tr|td|--|div|--|p|--|h1|h2|h3|h4|h5|h6|--|span|', # 显现这些元素
    '_elements_too_much_content': 	'|b|i|em|strong|--|h1|h2|h3|h4|h5|--|td|',              # 这里有多太重要内容
    '_elements_container': 			'|body|--|article|section|--|div|--|td|--|li|--|dd|dt|',        # 这些容器是装内容的
    '_elements_link_density':		'|div|--|table|ul|ol|--|section|aside|header|',     # 元素连结密度
    '_elements_floating':			'|div|--|table|',  # 元素浮动
    '_elements_above_target':		'|br|--|ul|ol|dl|',#上述目标元素
    '_elements_keep_attributes':                      # 介些属性保留
    {
        'a': 		['href', 'title', 'name'],
        'img': 		['src', 'width', 'height', 'alt', 'title'],

        'video': 	['src', 'width', 'height', 'poster', 'audio', 'preload', 'autoplay', 'loop', 'controls'],
        'audio': 	['src', 'preload', 'autoplay', 'loop', 'controls'],
        'source': 	['src', 'type'],

        'object': 	['data', 'type', 'width', 'height', 'classid', 'codebase', 'codetype'],
        'param': 	['name', 'value'],
        'embed': 	['src', 'type', 'width', 'height', 'flashvars', 'allowscriptaccess', 'allowfullscreen', 'bgcolor'],

        'iframe':	['src', 'width', 'height', 'frameborder', 'scrolling'],

        'td':		['colspan', 'rowspan'],
        'th':		['colspan', 'rowspan']
    }
}
# 要跳过链接
skipStuffFromDomains__links = ['doubleclick.net',
    'fastclick.net',
    'adbrite.com',
    'adbureau.net',
    'admob.com',
    'bannersxchange.com',
    'buysellads.com',
    'impact-ad.jp',
    'atdmt.com',
    'advertising.com',
    'itmedia.jp',
    'microad.jp',
    'serving-sys.com',
    'adplan-ds.com']

# 要跳过的图像
skipStuffFromDomain__images = [
    'googlesyndication.com',
    'fastclick.net',
    '.2mdn.net',
    'de17a.com',
    'content.aimatch.com',
    'bannersxchange.com',
    'buysellads.com',
    'impact-ad.jp',
    'atdmt.com',
    'advertising.com',
    'itmedia.jp',
    'microad.jp',
    'serving-sys.com',
    'adplan-ds.com']




def ss():
    _tag_name = 'ss'
    s = (parsingOptions['_elements_container'].find(''.join(['|', _tag_name, '|'])) > -1)
    print(s)

def s1():
    for link in skipStuffFromDomains__links:
        print(link)


def selGetSelection(inputStrWin):
    if(inputStrWin):
        ''
    else:
        return False

def measureText__getTextLength(_the_text):
    _text = _the_text
    return len(_text.decode('utf-8'))


def getContent__exploreNodeAndGetStuff(inputStrWin, _justExploring = False):
    ss = 0

    _global__element_index = 0

    _global__inside_link = False
    _global__inside_link__element_index = 0

    _global__length__above_plain_text = 0
    _global__count__above_plain_words = 0
    _global__length__above_links_text = 0      # 链接文本长度
    _global__count__above_links_words = 0      # 有多少链接纯文本
    _global__count__above_candidates = 0
    _global__count__above_containers = 0
    _global__above__plain_text = ''
    _global__above__links_text = ''

    _return__containers = []
    _return__candidates = []
    _return__links = []

    def _recursive(_node):
        print(ss)
        ss += 1
        return True
        dom = parseString(_node)
        _global__element_index = _global__element_index + 1
        _node = dom.getElementsByTagName('body')[0]    # test
#        _tag_name = _node.nodeType == 1 ? '#text' : ''
        _nodeType = _node.nodeType
        if _nodeType == 3 :
            _tag_name = '#text'
        else:
            if _nodeType == 1 and _node.nodeName and _node.nodeName > '':
                _tag_name = _node.nodeName.lower()
            else:
                _tag_name = '#invalid'

        _result = {
            '__index' : _global__element_index,
            '__node' : _node,

            '_is__container' : (parsingOptions['_elements_container'].find(''.join(['|', _tag_name, '|'])) > -1),
            '_is__candidate' : False,
            '_is__text' : False,
            '_is__link' : False,
            '_is__link_skip' : False,
            '_is__image_small' : False,
            '_is__image_medium' : False,
            '_is__image_large' : False,
            '_is__image_skip' : False,

# debug
            '_length__above_plain_text' : _global__length__above_plain_text,
            '_count__above_plain_words' : _global__count__above_plain_words,

            '_length__above_links_text' : _global__length__above_links_text,
            '_count__above_links_words' : _global__count__above_links_words,

            '_length__above_all_text' : (_global__length__above_plain_text + _global__length__above_links_text),
            '_count__above_all_words' : (_global__count__above_plain_words + _global__count__above_links_words),

            '_count__above_candidates' : _global__count__above_candidates,
            '_count__above_containers' : _global__count__above_containers,

            '_length__plain_text' : 0,
            '_count__plain_words' : 0,

            '_length__links_text' : 0,
            '_count__links_words' : 0,

            '_length__all_text' : 0,
            '_count__all_words' : 0,

            '_count__containers' : 0,
            '_count__candidates' : 0,

            '_count__links' : 0,
            '_count__links_skip' : 0,

            '_count__images_small' : 0,
            '_count__images_medium' : 0,
            '_count__images_large' : 0,
            '_count__images_skip' : 0
        }

#        第一波
        if _tag_name == '#invalid' :
            return False
        elif parsingOptions['_elements_ignore'].find(''.join(['|', _tag_name , '|'])) > -1:
            return False
        elif parsingOptions['_elements_visible'].find(''.join(['|', _tag_name , '|'])) > -1:
            ''
#            判断元素的位置
        elif parsingOptions['_elements_self_closing'].find(''.join(['|', _tag_name , '|'])) > -1:
            if _tag_name == 'img':
                ''
            else :
                return False
            ''

#        第二波
        if _tag_name == '#text':
            _result['_is__text'] = True

            _nodeText = _node.nodeValue
            _result['_length__plain_text'] = measureText__getTextLength(_nodeText)
            _result['_count__plain_words'] = measureText__getTextLength(_nodeText)  # 修改1 measureText__getWordCount

            if _global__inside_link :
                _global__length__above_links_text += _result['_length__plain_text']
                _global__count__above_links_words += _result['_count__plain_words']
#                if (false && $R.debug) { _global__above__links_text += ' ' + _nodeText; }
            else:
                _global__length__above_plain_text += _result['_length__plain_text']
                _global__count__above_plain_words += _result['_count__plain_words']
            return _result

        elif _tag_name == 'a':
            _href = _node.href
            if _href > '':                  # if (_href.indexOf); else { break; }
                _result['_is__link'] = True

                if isSkipStuffFroDomainsLinks(_href):
                    _result['_is__link_skip'] = True
                else:
                    if _global__inside_link:
                        ''
                    else:
                        _global__inside_link = True
                        _global__inside_link__element_index = _result['__index']

#                   完成
                    _return__links.append(_result)
        elif _tag_name == 'img':
            _src = _tag_name.src
            if _src:
                if isSkipStuffFromDomainImages(_src) :
                    _result['_is__image_skip'] = True
#            SIZE
#               判断图像高宽。。

#       循环遍历其子节点
        for _child in _node.childNodes:
            _child_result = _recursive(_child)
            if _child_result :
                ''
            else :
                continue

            if _child_result['_is__link'] == 1 :
                _result['_count__links'] += _child_result['_count__links'] + 1
            else :
                _result['_count__links'] += _child_result['_count__links']
            if _child_result['_is__link_skip'] == 1 :
                _result['_count__links_skip'] += _child_result['_count__links_skip'] + 1
            else :
                _result['_count__links_skip'] += _child_result['_count__links_skip']

            if _child_result['_is__container'] == 1:
                _result['_count__containers'] += _child_result['_count__containers'] + 1
            else :
                _result['_count__containers'] += _child_result['_count__containers']
            if _child_result['_is__candidate'] == 1:
                _result['_count__candidates'] += _child_result['_count__candidates'] + 1
            else :
                _result['_count__candidates'] += _child_result['_count__candidates']

            _result['_length__all_text'] += _child_result['_length__plain_text'] + _child_result['_length__links_text']
            _result['_count__all_words'] += _child_result['_count__plain_words'] + _child_result['_count__links_words']

            if _result['_is__link'] :
                _result['_length__links_text'] += _child_result['_length__plain_text'] + _child_result['_length__links_text']
                _result['_count__links_words'] += _child_result['_count__plain_words'] + _child_result['_count__links_words']
            else :
                _result['_length__plain_text'] += _child_result['_length__plain_text']
                _result['_count__plain_words'] += _child_result['_count__plain_words']
                _result['_length__links_text'] += _child_result['_length__links_text']
                _result['_count__links_words'] += _child_result['_count__links_words']

        if _result['_is__link'] and _global__inside_link__element_index == _result['__index'] :
            _global__inside_link = False
            _global__inside_link__element_index = 0

        if _result['_is__container'] or (_result['__index'] == 1 and _justExploring == True) :
            _return__containers.append(_result)
            if _result['_is__container'] :
                _global__count__above_containers += 1

            if _justExploring :
                ''
            else :
                if _result['_count__links'] * 2 >= _result['_count__plain_words'] :
                    ''
                elif _result['_length__plain_text'] < (65 / 3) :
                    ''
                elif _result['_count__plain_words'] < 5 :
                    ''
                elif _result['_length__plain_text'] < 10 :
                    ''
                elif _result['_count__plain_words'] < 2 :
                    ''
                else :
                    _result['_is__candidate'] = True
                    _return__candidates.append(_result)
                    _global__count__above_candidates += 1

            if _result['__index'] == 1 and not(_result['_is__candidate']) :
                _result['_is__candidate'] = True
                _result['_is__bad'] = True

                _return__candidates.append(_result)

        return _result

    _recursive(inputStrWin)
    if _justExploring :
        return _return__containers.pop()

    return {
        '_containers': 	_return__containers,    #容器
    '_candidates': 	_return__candidates,        #候选人
    '_links': 		_return__links
    }

def getContent__processCandidates(_candidatesToProcess):
    _candidates = _candidatesToProcess

#print(getContent__exploreNodeAndGetStuff(string))

# 判断是不是需要路过的链接
def isSkipStuffFroDomainsLinks(link):
    for skipLink in skipStuffFromDomains__links:
        if link.find(skipLink) > 0:
            return True
        else:
            return False

def isSkipStuffFromDomainImages(img):
    for skipImg in skipStuffFromDomain__images:
        if img.find(skipImg) > 0:
            return True
        else:
            return False



ss = 0
def xx():
    global ss
    ss += 1
    print(ss)

xx()

print(ss)



